import asyncio
from pyppeteer import launch
from pyquery import PyQuery as pq

# 起始页
url = 'https://eth.bitaps.com/blocks?page=1'
file_json = "res03.json"
# 最大采集页数，注意 当前看到是总共有65页
pagenum = 3
# 起始页数
current_page = 1


async def get_info():
    global pagenum
    global current_page
    browser = await launch(headless='', ignoreHTTPSErrors=True,
                           args=['--disable-infobars', '--window-size={1920},{1080}'])
    page = await browser.newPage()
    # 第一次执行，之后循环
    await page.goto(url, {'timeout': 10000 * 20})
    # 防止被识别
    await page.evaluateOnNewDocument('Object.defineProperty(navigator,"webdriver", {get: () => undefined})')
    while True:
        if current_page > pagenum:
            break
        current_page += 1
        # 开始采集
        print("当前页面是第{}页".format(current_page))
        await page.waitForSelector(".timeline")
        doc = pq(await page.content())
        kv_li_list = []
        for li in doc(".timeline").items():
            time = li(".time-date .bold").text()
            # print("time:\n{}".format(time))
            for item in doc(".tml-table-wrap").items():
                value_list_tr = []
                key_list_tr = []
                for tr in item("tr").items():
                    key = tr("td:nth-child(1)").text()
                    value = tr("td").text().strip(key).strip()
                    key_list_tr.append(key)
                    value_list_tr.append(value)
                kv_li_list += (list(zip(key_list_tr, value_list_tr)))
        print("kv_li_list:\n{}".format(kv_li_list))

        with open(file_json, mode='a+') as f:
            # for kv_li_list in get_info():
            f.write(str(kv_li_list))
        # 点击下一页  注意 这个按钮本身功能有问题
        await page.click('li:last-child', {'timeout': 10000 * 20})
        await page.waitForSelector(".timeline")


if __name__ == '__main__':
    # 执行之前先清理上次数据
    with open(file_json, 'w', encoding='utf-8') as f:
        f.write("")
    # 起协程loop执行协程函数
    asyncio.get_event_loop().run_until_complete(get_info())

